#!/usr/bin/python
from urllib2 import urlopen
from urllib2 import build_opener
from bs4 import BeautifulSoup, SoupStrainer
from pymongo import MongoClient
from bson.objectid import ObjectId
import time
from threading import Thread
import madsutil
from madsutil import log
from madsutil import logt
from madsutil import url_fix
import re


def test():
    print 'test'



def getnextpag(pagination):
    '''retrieve the next link on the pagination div of the page if exist'''
    for link in pagination.findAll('a'):
        if link.string.strip() == 'Next':
            return link
    
    return None

def get_cards(pagn=1,elements=10):
    '''
    card pagination
    pagn is the n-th page
    elements is the number of element for page
    '''
    client = MongoClient(madsutil.DB_URL)
    db = client[madsutil.DB_NAME]
    cards = db.cards
    
    #q = cards.find({},{'_id':1,'name':1})
    q = cards.find().skip((pagn-1)*elements).limit(elements);
    allcard=[]
    for c in q:
        allcard.append(c)
        print type(c),c['name']
         
    log('Card count: '+str(cards.count()))
    return allcard
    

def loads_detail(card):
    '''open the card link,parse the card detail from 
    gatherer.wizards.com, and add the founded attribute to selected
    card and return it'''
    start_det_time = time.time()
    opener = build_opener()
    opener.addheaders = [('User-agent', 'Mozilla/5.0')]
    html = opener.open(card['link'].encode('utf-8')).read()
    print 'open link: ' + card['link'].encode('utf-8')
    #html = urlopen(card['link'].encode('utf-8')).read()
    logt('url http://deckbox.org open in',start_det_time)
    divparse = SoupStrainer('div',{'class': 'card_page'})
    elapse = time.time()
    page = BeautifulSoup(html,parse_only=divparse)
    logt('Parse http://deckbox.org content in',elapse)
    price_min = page.find("div", {"class" : "price price_min"}).contents[0].string.strip()
    price_avg = page.find("div", {"class" : "price price_avg"}).contents[0].string.strip()
    price_hig = page.find("div", {"class" : "price price_hig"}).contents[0].string.strip()
    
    prices = {'price_min':price_min,
              'price_avg':price_avg,
              'price_hig':price_hig};
    #print prices
    card['prices']=prices
    
   
    
            
        
        
    card['img_link'] = url_fix('http://deckbox.org'+page.find("img", {"id": "card_image"})['src'] )
    
    cardprop = page.find("table", {"class" : "card_properties"})
    elapse = time.time()
    for row in cardprop.findAll('tr'):
        cols = row.findAll('td');
        for c in cols:
            if c.contents[0] == 'Editions':
                images = cols[1].findAll('img')
                editions = []
                for img in images:
                    editions.append(img['alt'])
                card['editions']=editions
            if c.contents[0] == 'Formats':
                card['formats'] = cols[1].contents[0].strip();
            if c.contents[0] == 'Rulings':
                card['gatherer_page'] = url_fix(cols[1].find('a')['href'])
            if c.contents[0] == 'Flavor':
                card['flavor'] = cols[1].contents[0].strip();
    logt('retrieve all from deckbox in',elapse)
        
    '''find other attributes on http://gatherer.wizards.com '''
    elapse = time.time()
    html = urlopen(card['gatherer_page'].encode('utf-8')).read()
    logt('url gatherer open in',start_det_time)
    tablesparse = SoupStrainer('div',{"class":"contentcontainer"})
    #tablesparse = SoupStrainer('td',{"class":"rightCol"})
    elapse = time.time()
    page = BeautifulSoup(html,parse_only=tablesparse)
    madsutil.logt('Parse gatherer page content in',elapse)
    _double = False;
    #cxheck if it is a double card
    tables = page.findAll("table",{'class':'cardDetails'})
    if len(tables)>1:
        _double = True
    
    card['double'] = _double

    

    
    elapse = time.time()
    '''Carta principale'''
    divs = tables[0].findAll("div",{'class':'row'})
    for d in divs:
        #card[d.find('div',{'class':'label'}).text.strip()].append(d.find('div',{'class':'value'}))
        if d.find('div',{'class':'label'}).text.strip() == 'Card Name:':
            name_found = d.find('div',{'class':'value'}).text.strip()
            if(name_found != card['name']):
                log('ERROR: parsing the wrong page')
                    
        if d.find('div',{'class':'label'}).text.strip() == 'Mana Cost:':
            images = d.find("div",{"class":"value"}).findAll('img');
            manas = []
            for img in images:
                manas.append(img['alt'])
            #print manas
            card['mana']=manas
            continue
            
        if d.find('div',{'class':'label'}).text.strip() == 'Converted Mana Cost:':
            cmc = d.find('div',{'class':'value'}).text.strip()
            card['cmc'] = cmc
            continue
        
        if d.find('div',{'class':'label'}).text.strip() == 'Types:':
            typ = d.find('div',{'class':'value'}).text.strip()
            card['type'] = typ
            continue
        
        if d.find('div',{'class':'label'}).text.strip() == 'Card Text:':
            text_value = d.find("div",{'class':'value'}).findAll("div",{'class':'cardtextbox'})
            rules = ''
            for divs in text_value:
                images = divs.findAll('img')
                if images:
                    for img in images:
                        rules = rules + str(img['alt'])+','
                rules = rules + divs.text.strip()+'\n'
            card['rules'] = rules
            continue
        
        if d.find('div',{'class':'label'}).text.strip() == 'P/T:':
            pt = d.find('div',{'class':'value'}).text.strip()
            card['p'] = pt.split('/')[0].strip()
            card['t'] = pt.split('/')[1].strip()
            continue
    
        if d.find('div',{'class':'label'}).text.strip() == 'Expansion:':
            expansion = d.find('div',{'class':'value'}).text.strip()
            card['expansion'] = expansion
            continue
    
        if d.find('div',{'class':'label'}).text.strip() == 'Rarity:':
            rarity = d.find('div',{'class':'value'}).text.strip()
            card['rarity'] = rarity
            continue
        
        if d.find('div',{'class':'label'}).text.strip() == 'Community Rating:':
            rating = d.find('div',{'class':'value'}).text.strip()
            card['rating'] = rating[rating.index(':')+1:rating.index('/')]
            continue
    
        if d.find('div',{'class':'label'}).text.strip() == 'Color Indicator:':
            color = d.find('div',{'class':'value'}).text.strip()
            card['color'] = color
            continue
    '''Gestione carta doppia'''
    
    if _double:
        divs = tables[1].findAll("div",{'class':'row'})
        for d in divs:
            
            if d.find('div',{'class':'label'}).text.strip() == 'Mana Cost:':
                images = d.find("div",{"class":"value"}).findAll('img');
                manas = []
                for img in images:
                    manas.append(img['alt'])
                #print manas
                card['mana_d'] = manas
                continue
                
            if d.find('div',{'class':'label'}).text.strip() == 'Converted Mana Cost:':
                cmc = d.find('div',{'class':'value'}).text.strip()
                card['cmc_d'] = cmc
                continue
            
            if d.find('div',{'class':'label'}).text.strip() == 'Types:':
                typ = d.find('div',{'class':'value'}).text.strip()
                card['type_d'] = typ
                continue
            
            if d.find('div',{'class':'label'}).text.strip() == 'Card Text:':
                text_value = d.find("div",{'class':'value'}).findAll("div",{'class':'cardtextbox'})
                rules = ''
                for divs in text_value:
                    images = divs.findAll('img')
                    if images:
                        for img in images:
                            rules = rules + str(img['alt'])+','
                    rules = rules + divs.text.strip()+'\n'
                card['rules_d'] = rules
                continue
            
            if d.find('div',{'class':'label'}).text.strip() == 'P/T:':
                pt = d.find('div',{'class':'value'}).text.strip()
                card['p_d'] = pt.split('/')[0].strip()
                card['t_d'] = pt.split('/')[1].strip()
                #card['pt_d'].append(pt)
                continue
            
            if d.find('div',{'class':'label'}).text.strip() == 'Color Indicator:':
                color = d.find('div',{'class':'value'}).text.strip()
                card['color_d'] = color
                continue
            
    logt('end tables routines in',elapse)
    logt('end card in',start_det_time)
    return card

    
    

def load_cards(start_from_last=False, update=False, appender=None):
    ''' 
        Read info from deckbox.org and gatherer.wizards.com and insert
        cards information on DB
        if start_from_last=True search for the last inserted card before
        start inserting
        if update=true update a card if is into db
    '''

    try:
        
        log('['+str(time.time())+'] Start loading card [start_from_last='+str(start_from_last)+'] [update='+str(update)+']',appender)
        
        client = MongoClient(madsutil.DB_URL)
        db = client[madsutil.DB_NAME]
        cards = db.cards
        
        if (start_from_last):
            '''elimino l'ultima carta che potrebbe essere in uno stato inconsistente'''
            last_card = cards.find().sort(  [('_id',-1 )] ).limit(1).next();
            log('Last Card Inserted: ' + last_card['name'],appender)
            cards.remove({'name':last_card['name']})
            log('Remove Card: ' + last_card['name'],appender)
        
        #detailurl = "http://gatherer.wizards.com/Pages/Card/Details.aspx?multiverseid=1"
        deckboxurl = "http://deckbox.org/games/mtg/cards"
        
        tablesparse = SoupStrainer('div',{'id': 'set_cards_table'})
        html = urlopen(deckboxurl).read()
        page = BeautifulSoup(html,parse_only=tablesparse)
        
        cardtable = page.find("table", {"class" : "set_cards simple_table main full"})
        pagination = page.find("div", {"class" : "pagination_controls"});
    
        lastCard = {}
        if (start_from_last):
            lastCard = cards.find().sort(  [('_id',-1 )] ).limit(1).next();
            log('Last Card to find: ' + lastCard['name'],appender)
            
        _header = 0;
        start_time = time.time()
        _page = 1;
        while getnextpag(pagination):
            start_time_page = time.time()
            for row in cardtable.findAll('tr'):
                if(_header<2):
                    _header+=1
                    continue
                try:
                    cols = row.findAll('td')
                    name = cols[0].find('a').contents[0].string.strip().encode()
                    link = url_fix(cols[0].find('a')['href'])
                    card = {"name":name,
                            "link":link}
                    
                    if (start_from_last):
                        if card['name'] == lastCard['name']:
                            log('Last Card Inserted: ' +card['name'],appender)
                            start_from_last=False
                            continue
                        else:
                            continue
    
                    founded = cards.find_one({"name": card['name']})
                    if not founded:
                        card = loads_detail(card)
                        card_id = cards.insert(card);
                        log( 'Insert Card[' +str(card['name'])+'] with id: ' + str(ObjectId(card_id)) , appender)
                    elif update:
                        founded = loads_detail(founded)
                        cards.save(founded)
                        log( 'Card updated: '+ str(founded['name']),appender)
                    else:
                        log( 'Card yet present on db: '+ str(founded['name']),appender)
                    
                except Exception as e:
                    log('Card name: ' + card['name'],appender)
                    log( 'Card link: ' + card['link'],appender)
                    log( "unable to parse this card: "+str(e),appender)
                    log(str(card),appender)
                    raise
                    
            
            _header=0
            deckboxurl = "http://deckbox.org"+getnextpag(pagination)['href']
            html = urlopen(deckboxurl).read()
            page = BeautifulSoup(html,parse_only=tablesparse)
            cardtable = page.find("table", {"class" : "set_cards simple_table main full"})
            pagination = page.find("div", {"class" : "pagination_controls"});
            logt('Page '+str(_page)+ ' read',start_time_page)
            _page +=1
    
        client.close()
        madsutil.log('All cards reads',start_time,appender)
    except KeyboardInterrupt:
        lastCard = cards.find().sort( { '_id' : -1 } ).limit(1).next();
        cards.remove({"_id":ObjectId(lastCard._id)})
        exit(0)
    except SystemExit:
        lastCard = cards.find().sort( { '_id' : -1 } ).limit(1).next();
        cards.remove({"_id":ObjectId(lastCard._id)})
        exit(0)
    
    
    
def load_keywords(appender=None):
    client = MongoClient(madsutil.DB_URL)
    db = client[madsutil.DB_NAME]
    keywords = db.keywords
    
    keywordsURL = "http://en.wikipedia.org/wiki/List_of_Magic:_The_Gathering_keywords"
    
    tablesparse = SoupStrainer('div',{'id': 'mw-content-text'})
    html = urlopen(keywordsURL).read()
    page = BeautifulSoup(html,parse_only=tablesparse)

    headers = page.findAll("h3")
    
    for header in headers:
        keyword = {'name':header.find('span',{'class':'mw-headline'}).string}
        if not keywords.find_one(keyword):
            keywords.insert(keyword);
            log('keyword: ' + keyword['name'] + ' inserted',appender)
        else:
            log('keyword: ' + keyword['name'] + ' yet present on db',appender)
            time.sleep(0.01)
        
    log('Done!',appender)


class LoadKeyWordsWorker(Thread):
    '''
    This working class execute the load_key_words methods
    as a separate thread
    '''
    
    def __init__(self, appender=None):
        super(LoadKeyWordsWorker, self).__init__()
        self._appender = appender
    
    def run(self):
        load_keywords(self._appender)

class LoadCardsWorker(Thread):
    '''
    This working class execute the load_cards methods
    as a separate thread
    '''    
    def __init__(self, start_from_last=False,update=False,appender=None):
        super(LoadCardsWorker, self).__init__()
        self._appender = appender
        self._start_from_last = start_from_last
        self._update=update
    
    def setAppender(self,appender):
        self._appender = appender
    
    def run(self):
        load_cards(self._start_from_last, self._update, self._appender)
        
    
    
def link_abilities():
    client = MongoClient(madsutil.DB_URL)
    db = client[madsutil.DB_NAME]
    cards = db.cards
    keywords = db.keywords
    #setta l'attributo sul db
    cards.update({}, {'$set': {'abilities': ''}}, upsert=False, multi=True)

    for keyword in keywords.find():
        keywordValue = keyword['name']
        cardsWithAbility = cards.find({'rules': re.compile(keywordValue, re.IGNORECASE)})
        #print(keywordValue)
        for cardWithAbility in cardsWithAbility:
            #se la parola chiave e' nel nome e il nome e' contenuto nel testo e la parola appare meno del
            #nome non la considero
            if(cardWithAbility['name'].count(keywordValue) > 0 and
                    cardWithAbility['rules'].count(keywordValue) < cardWithAbility['rules'].count(cardWithAbility['name'])):
                continue
            print 'Found rule in card: '+cardWithAbility['name']
            cardAbilities = cardWithAbility['abilities']
            if(not cardAbilities):
                cardAbilities = []
            if(not keywordValue in cardAbilities):
                cardAbilities.append(keywordValue)
                cardWithAbility['abilities'] = cardAbilities
                cards.save(cardWithAbility)



    
    
    
    
    
    
    
    
    
    
    